!pip install pandas 
!pip install numpy 
!pip install seaborn 
!pip install scipy 
!pip install requests 
!pip install scikit-learn 
!pip install statsmodels 
!pip install matplotlib
!pip install xgboost

Requirement already satisfied: pandas in ./.venv/lib/python3.12/site-packages (2.3.3)
Requirement already satisfied: numpy>=1.26.0 in ./.venv/lib/python3.12/site-packages (from pandas) (2.4.1)
Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.12/site-packages (from pandas) (2.9.0.post0)
Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.12/site-packages (from pandas) (2025.2)
Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.12/site-packages (from pandas) (2025.3)
Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)
Requirement already satisfied: numpy in ./.venv/lib/python3.12/site-packages (2.4.1)
Requirement already satisfied: seaborn in ./.venv/lib/python3.12/site-packages (0.13.2)
Requirement already satisfied: numpy!=1.24.0,>=1.20 in ./.venv/lib/python3.12/site-packages (from seaborn) (2.4.1)
Requirement already satisfied: pandas>=1.2 in ./.venv/lib/python3.12/site-packages (from seaborn) (2.3.3)
Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in ./.venv/lib/python3.12/site-packages (from seaborn) (3.10.8)
Requirement already satisfied: contourpy>=1.0.1 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.3.3)
Requirement already satisfied: cycler>=0.10 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.61.1)
Requirement already satisfied: kiwisolver>=1.3.1 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.9)
Requirement already satisfied: packaging>=20.0 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (25.0)
Requirement already satisfied: pillow>=8 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (12.1.1)
Requirement already satisfied: pyparsing>=3 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.3.2)
Requirement already satisfied: python-dateutil>=2.7 in ./.venv/lib/python3.12/site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.9.0.post0)
Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.12/site-packages (from pandas>=1.2->seaborn) (2025.2)
Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.12/site-packages (from pandas>=1.2->seaborn) (2025.3)
Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.12/site-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.17.0)
Requirement already satisfied: scipy in ./.venv/lib/python3.12/site-packages (1.17.0)
Requirement already satisfied: numpy<2.7,>=1.26.4 in ./.venv/lib/python3.12/site-packages (from scipy) (2.4.1)
Requirement already satisfied: requests in ./.venv/lib/python3.12/site-packages (2.32.5)
Requirement already satisfied: charset_normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests) (3.4.4)
Requirement already satisfied: idna<4,>=2.5 in ./.venv/lib/python3.12/site-packages (from requests) (3.11)
Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests) (2.6.3)
Requirement already satisfied: certifi>=2017.4.17 in ./.venv/lib/python3.12/site-packages (from requests) (2026.1.4)
Requirement already satisfied: scikit-learn in ./.venv/lib/python3.12/site-packages (1.8.0)
Requirement already satisfied: numpy>=1.24.1 in ./.venv/lib/python3.12/site-packages (from scikit-learn) (2.4.1)
Requirement already satisfied: scipy>=1.10.0 in ./.venv/lib/python3.12/site-packages (from scikit-learn) (1.17.0)
Requirement already satisfied: joblib>=1.3.0 in ./.venv/lib/python3.12/site-packages (from scikit-learn) (1.5.3)
Requirement already satisfied: threadpoolctl>=3.2.0 in ./.venv/lib/python3.12/site-packages (from scikit-learn) (3.6.0)
Requirement already satisfied: statsmodels in ./.venv/lib/python3.12/site-packages (0.14.6)
Requirement already satisfied: numpy<3,>=1.22.3 in ./.venv/lib/python3.12/site-packages (from statsmodels) (2.4.1)
Requirement already satisfied: scipy!=1.9.2,>=1.8 in ./.venv/lib/python3.12/site-packages (from statsmodels) (1.17.0)
Requirement already satisfied: pandas!=2.1.0,>=1.4 in ./.venv/lib/python3.12/site-packages (from statsmodels) (2.3.3)
Requirement already satisfied: patsy>=0.5.6 in ./.venv/lib/python3.12/site-packages (from statsmodels) (1.0.2)
Requirement already satisfied: packaging>=21.3 in ./.venv/lib/python3.12/site-packages (from statsmodels) (25.0)
Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.12/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2.9.0.post0)
Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.12/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2025.2)
Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.12/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2025.3)
Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas!=2.1.0,>=1.4->statsmodels) (1.17.0)
Requirement already satisfied: matplotlib in ./.venv/lib/python3.12/site-packages (3.10.8)
Requirement already satisfied: contourpy>=1.0.1 in ./.venv/lib/python3.12/site-packages (from matplotlib) (1.3.3)
Requirement already satisfied: cycler>=0.10 in ./.venv/lib/python3.12/site-packages (from matplotlib) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in ./.venv/lib/python3.12/site-packages (from matplotlib) (4.61.1)
Requirement already satisfied: kiwisolver>=1.3.1 in ./.venv/lib/python3.12/site-packages (from matplotlib) (1.4.9)
Requirement already satisfied: numpy>=1.23 in ./.venv/lib/python3.12/site-packages (from matplotlib) (2.4.1)
Requirement already satisfied: packaging>=20.0 in ./.venv/lib/python3.12/site-packages (from matplotlib) (25.0)
Requirement already satisfied: pillow>=8 in ./.venv/lib/python3.12/site-packages (from matplotlib) (12.1.1)
Requirement already satisfied: pyparsing>=3 in ./.venv/lib/python3.12/site-packages (from matplotlib) (3.3.2)
Requirement already satisfied: python-dateutil>=2.7 in ./.venv/lib/python3.12/site-packages (from matplotlib) (2.9.0.post0)
Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.12/site-packages (from python-dateutil>=2.7->matplotlib) (1.17.0)
Requirement already satisfied: xgboost in ./.venv/lib/python3.12/site-packages (3.2.0)
Requirement already satisfied: numpy in ./.venv/lib/python3.12/site-packages (from xgboost) (2.4.1)
Requirement already satisfied: nvidia-nccl-cu12 in ./.venv/lib/python3.12/site-packages (from xgboost) (2.29.3)
Requirement already satisfied: scipy in ./.venv/lib/python3.12/site-packages (from xgboost) (1.17.0)

import pandas as pd
import numpy as np
import seaborn as sns
import scipy.stats as stats
import requests
import math

from sklearn.model_selection import train_test_split
from sklearn.impute import KNNImputer
from sklearn.linear_model import LinearRegression, Ridge, RidgeCV, Lasso, LassoCV
from sklearn.metrics import mean_absolute_error,r2_score,mean_squared_error
from sklearn.metrics import roc_auc_score
from scipy.stats import shapiro
from scipy.stats import chi2_contingency
from scipy.stats import shapiro, ttest_ind, mannwhitneyu, f_oneway, kruskal, spearmanr
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from statsmodels.stats.outliers_influence import variance_inflation_factor
import statsmodels.api as sm

import matplotlib.pyplot as plt

dataframe = pd.read_csv("life_expectancy_data.csv")
dataframe.columns = dataframe.columns.str.strip()
dataframe.head()

dataframe = dataframe.replace(" ",np.nan)

dataframe.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2938 entries, 0 to 2937
Data columns (total 22 columns):
 #   Column                           Non-Null Count  Dtype  
---  ------                           --------------  -----  
 0   Country                          2938 non-null   object 
 1   Year                             2938 non-null   int64  
 2   Status                           2938 non-null   object 
 3   Life expectancy                  2928 non-null   float64
 4   Adult Mortality                  2928 non-null   float64
 5   infant deaths                    2938 non-null   int64  
 6   Alcohol                          2744 non-null   float64
 7   percentage expenditure           2938 non-null   float64
 8   Hepatitis B                      2385 non-null   float64
 9   Measles                          2938 non-null   int64  
 10  BMI                              2904 non-null   float64
 11  under-five deaths                2938 non-null   int64  
 12  Polio                            2919 non-null   float64
 13  Total expenditure                2712 non-null   float64
 14  Diphtheria                       2919 non-null   float64
 15  HIV/AIDS                         2938 non-null   float64
 16  GDP                              2490 non-null   float64
 17  Population                       2286 non-null   float64
 18  thinness 10-19 years             2904 non-null   float64
 19  thinness 5-9 years               2904 non-null   float64
 20  Income composition of resources  2771 non-null   float64
 21  Schooling                        2775 non-null   float64
dtypes: float64(16), int64(4), object(2)
memory usage: 505.1+ KB

dataframe.describe(include="all").T

dataframe[["Life expectancy"]].describe().T.join(
    pd.DataFrame({
    "median" : [dataframe["Life expectancy"].median()]
    },index=["Life expectancy"])
)

plt.figure(figsize=(8, 5))
plt.hist(dataframe["Life expectancy"], bins=30,edgecolor="black",linewidth=1)
plt.xlabel("Life expectancy (godine)")
plt.ylabel("Broj osoba")
plt.title("Distribucija Life Expectancy")
plt.show()

plt.figure(figsize=(8, 5))
plt.hist(dataframe['GDP'],edgecolor="black",linewidth=1)
plt.xlabel("GDP")
plt.ylabel("Broj obeležja")
plt.title("Distribucija GDP")
plt.show()

plt.figure(figsize=(8, 5))
plt.hist(dataframe['HIV/AIDS'],edgecolor="black",linewidth=1)
plt.xlabel("HIV/AIDS")
plt.ylabel("Broj")
plt.title("Distribucija HIV/AIDS")
plt.show()

plt.figure(figsize=(8, 5))
plt.hist(dataframe['BMI'],edgecolor="black",linewidth=1)
plt.xlabel("BMI")
plt.ylabel("Broj")
plt.title("Distribucija BMI")
plt.show()

plt.figure(figsize=(8, 5))
plt.hist(dataframe['percentage expenditure'],edgecolor="black",linewidth=1)
plt.xlabel("percentage expenditure")
plt.ylabel("Broj")
plt.title("Distribucija percentage expenditure")
plt.show()

plt.figure(figsize=(8, 5))
plt.hist(dataframe['thinness 10-19 years'],edgecolor="black",linewidth=1)
plt.xlabel("thinness 10-19 years")
plt.ylabel("Broj")
plt.title("Distribucija thinness 10-19 years")
plt.show()

uniques = dataframe["Country"].nunique()

print("Broj jedinstvenih država u datasetu :",uniques)

Broj jedinstvenih država u datasetu : 193

top_countries = dataframe["Country"].value_counts().head(15).index

dataframe[dataframe["Country"].isin(top_countries)].boxplot(
    column="Life expectancy",
    by="Country",
    figsize=(10, 6),
    rot=45
)

plt.title("Life Expectancy grupisan po Country")
plt.suptitle("")
plt.grid(False)
plt.show()

years = dataframe["Year"].value_counts().index

dataframe[dataframe["Year"].isin(years)].boxplot(
    column="Life expectancy",
    by="Year",
    figsize=(10, 6),
    rot=45
)

plt.title("Life Expectancy grupisan po Year")
plt.suptitle("")
plt.grid(False)
plt.show()

dataframe["Status"].unique()

array(['Developing', 'Developed'], dtype=object)

status = dataframe["Status"].value_counts().index

dataframe[dataframe["Status"].isin(status)].boxplot(
    column="Life expectancy",
    by="Status",
    figsize=(10, 6),
    rot=45
)

plt.title("Life Expectancy grupisan po Status")
plt.suptitle("")
plt.grid(False)
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Adult Mortality"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Adult Mortality")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Adult Mortality by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["infant deaths"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("infant deaths")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs infant deaths by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["infant deaths"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlim(0,150)
plt.xlabel("infant deaths")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs infant deaths by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Alcohol"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Alcohol")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Alcohol by Status")
plt.legend()
plt.show()

filtered_df_alcohol = (
    dataframe.loc[dataframe["Alcohol"] >= 15,
                  ["Country","Alcohol"]]
    .sort_values(by="Alcohol", ascending=False)
)
filtered_df_alcohol

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["percentage expenditure"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("percentage expenditure")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs percentage expenditure by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["percentage expenditure"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlim(0,2500)
plt.xlabel("percentage expenditure")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs percentage expenditure by Status (<=2500)")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Hepatitis B"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Hepatitis B")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Hepatitis B (%) by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Measles"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Measles")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Measles by Status")
plt.legend()
plt.show()

filtered_df_measles = (
    dataframe.loc[dataframe["Measles"] >= 100000,
                  ["Country","Measles"]]
    .sort_values(by="Measles", ascending=False)
)
filtered_df_measles

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["BMI"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("BMI")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs BMI")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["under-five deaths"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("under-five deaths")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs under-five deaths by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Polio"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Polio")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Polio deaths by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Total expenditure"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Total expenditure")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Total Expenditure by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Diphtheria"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Diphtheria")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Diphtheria by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["HIV/AIDS"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("HIV/AIDS")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs HIV/AIDS by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["GDP"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("GDP")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs GDP by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["GDP"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xscale("log")
plt.xlabel("GDP")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs GDP by Status")
plt.legend()
plt.show()

filtered_df_GDP = (
    dataframe.loc[dataframe["GDP"] >= 60000,
                  ["Country","GDP"]]
    .sort_values(by="GDP", ascending=False)
)
filtered_df_GDP

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Population"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Population")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Population by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Population"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlim(0,40000000)
plt.xlabel("Population")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Population by Status")
plt.legend()
plt.show()

filtered_df_population = (
    dataframe.loc[dataframe["Population"] >= 1000000000, 
                  ["Country", "Population"]]
    .sort_values(by="Population", ascending=False)
)
filtered_df_population

filtered_df_china = dataframe.loc[
    dataframe["Country"] == "China",
    ["Country", "Population"]
]
filtered_df_china

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["thinness 10-19 years"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("thinness 1-19 years")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs thinness 1-19 years by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["thinness 5-9 years"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("thinness 5-9 years")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs thinness 5-9 years by Status")
plt.legend()
plt.show()

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Income composition of resources"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Income composition of resources")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Income composition of resources by Status")
plt.legend()
plt.show()

df_filtered_icr = (
    dataframe.loc[dataframe["Income composition of resources"] < 0.1,
                  ["Country","Income composition of resources"]]
    .sort_values(by="Income composition of resources", ascending=False)
    
)
df_filtered_icr

plt.figure(figsize=(10, 8))

for status in dataframe["Status"].unique():
    subset = dataframe[dataframe["Status"] == status]
    plt.scatter(
        subset["Schooling"],
        subset["Life expectancy"],
        alpha=0.4,
        label=status
    )

plt.xlabel("Schooling")
plt.ylabel("Life expectancy")
plt.title("Life Expectancy vs Schooling by Status")
plt.legend()
plt.show()

def spearman_test(df, x, y="Life expectancy"):
    tmp = df[[x, y]].dropna()
    r, p = spearmanr(tmp[x], tmp[y])
    print(f"{x} vs {y}")
    print(f"Spearman r = {r:.4f}")
    print(f"p-vrednost = {p:.6g}")
    return r, p

spearman_test(dataframe, "Adult Mortality")
spearman_test(dataframe, "HIV/AIDS")
spearman_test(dataframe, "Schooling")
spearman_test(dataframe, "GDP")
spearman_test(dataframe, "Income composition of resources")

Adult Mortality vs Life expectancy
Spearman r = -0.6500
p-vrednost = 0
HIV/AIDS vs Life expectancy
Spearman r = -0.7536
p-vrednost = 0
Schooling vs Life expectancy
Spearman r = 0.8135
p-vrednost = 0
GDP vs Life expectancy
Spearman r = 0.6425
p-vrednost = 2.35903e-289
Income composition of resources vs Life expectancy
Spearman r = 0.8657
p-vrednost = 0

(np.float64(0.8657015149223543), np.float64(0.0))

tmp = dataframe[["Status", "Life expectancy"]].dropna()

developed = tmp[tmp["Status"] == "Developed"]["Life expectancy"]
developing = tmp[tmp["Status"] == "Developing"]["Life expectancy"]

u_stat, p_value = mannwhitneyu(developed, developing, alternative="two-sided")

print(f"U statistika = {u_stat:.4f}")
print(f"p-vrednost = {p_value:.6g}")
print(f"Medijana (Developed) = {developed.median():.3f}")
print(f"Medijana (Developing) = {developing.median():.3f}")

U statistika = 1131521.0000
p-vrednost = 1.37111e-191
Medijana (Developed) = 79.250
Medijana (Developing) = 69.000

tmp = dataframe[["Year", "Life expectancy"]].dropna()
r_year, p_year = spearmanr(tmp["Year"], tmp["Life expectancy"])

print(f"Spearman r = {r_year:.4f}")
print(f"p-vrednost = {p_year:.6g}")

Spearman r = 0.1572
p-vrednost = 1.18955e-17

dataframe.duplicated().any()

np.False_

(dataframe.isnull().sum()/dataframe.shape[0]*100).round(2)

Country                             0.00
Year                                0.00
Status                              0.00
Life expectancy                     0.34
Adult Mortality                     0.34
infant deaths                       0.00
Alcohol                             6.60
percentage expenditure              0.00
Hepatitis B                        18.82
Measles                             0.00
BMI                                 1.16
under-five deaths                   0.00
Polio                               0.65
Total expenditure                   7.69
Diphtheria                          0.65
HIV/AIDS                            0.00
GDP                                15.25
Population                         22.19
thinness 10-19 years                1.16
thinness 5-9 years                  1.16
Income composition of resources     5.68
Schooling                           5.55
dtype: float64

df = (dataframe.isna().mean()*100).round(2)
df = df[df > 0].sort_values()

df.plot(kind="barh", figsize=(8,5), title="Nedostajuće vrednosti (%)")
plt.xlabel("%")
plt.tight_layout()
plt.show()

missing_data = dataframe.columns[dataframe.isna().any()]

miss = dataframe[missing_data].isna().astype(int)

corr = miss.corr()

results = []
for i in corr.columns:
    for j in corr.columns:
        if i < j:
            r = corr.loc[i, j]
            if r > 0.3:
                results.append((i, j, r))

results.sort(key=lambda x: x[2], reverse=True)

print("Korelacija nedostajucih vrednosti")
print("-" * 70)
for i, j, r in results:
    print(f"{i:32}    {j:32}  r={r:.3f}")

Korelacija nedostajucih vrednosti
----------------------------------------------------------------------
Adult Mortality                     Life expectancy                   r=1.000
BMI                                 thinness 10-19 years              r=1.000
BMI                                 thinness 5-9 years                r=1.000
Diphtheria                          Polio                             r=1.000
thinness 10-19 years                thinness 5-9 years                r=1.000
Income composition of resources     Schooling                         r=0.987
Alcohol                             Total expenditure                 r=0.895
GDP                                 Population                        r=0.744
GDP                                 Schooling                         r=0.559
GDP                                 Income composition of resources   r=0.554
Income composition of resources     Population                        r=0.456
Population                          Schooling                         r=0.454
BMI                                 Polio                             r=0.428
BMI                                 Diphtheria                        r=0.428
Polio                               thinness 10-19 years              r=0.428
Polio                               thinness 5-9 years                r=0.428
Diphtheria                          thinness 10-19 years              r=0.428
Diphtheria                          thinness 5-9 years                r=0.428

fig, axis = plt.subplots(figsize=(9,7))

heatmap = axis.imshow(corr, cmap="RdYlBu_r", vmin=-1, vmax=1)

axis.set_xticks(range(len(corr.columns)))
axis.set_yticks(range(len(corr.columns)))

axis.set_xticklabels(corr.columns, rotation=45, ha="right")
axis.set_yticklabels(corr.columns)

for i in range(len(corr)):
    for j in range(len(corr)):
        axis.text(j, i, round(corr.iloc[i, j], 2),
                ha="center", va="center", fontsize=7)

plt.colorbar(heatmap)

axis.set_title("Korelacija nedostajućih vrednosti")

plt.show()

dataframe["Population"].describe()

count    2.286000e+03
mean     1.275338e+07
std      6.101210e+07
min      3.400000e+01
25%      1.957932e+05
50%      1.386542e+06
75%      7.420359e+06
max      1.293859e+09
Name: Population, dtype: float64

plt.hist(dataframe["Population"].dropna(), bins=40, edgecolor="black",linewidth=1)
plt.title("Distribucija populacije")

plt.show()

pop = pd.to_numeric(dataframe["Population"], errors="coerce")

print(pop.describe())
print("Missing %:", pop.isna().mean()*100)

plt.figure(figsize=(6,4))
plt.hist(np.log10(pop.dropna()), bins=40, edgecolor="black",linewidth=1)
plt.title("Population distribution (log10 scale)")
plt.xlabel("log10(Population)")
plt.ylabel("Count")
plt.show()

count    2.286000e+03
mean     1.275338e+07
std      6.101210e+07
min      3.400000e+01
25%      1.957932e+05
50%      1.386542e+06
75%      7.420359e+06
max      1.293859e+09
Name: Population, dtype: float64
Missing %: 22.19196732471069

bad = (pop <= 0)
print("Broj redova sa vrednoscu 0:", bad.sum())

Broj redova sa vrednoscu 0: 0

miss_by_year = dataframe.groupby("Year")["Population"].apply(lambda s: s.isna().mean())

plt.figure(figsize=(7,3))
plt.plot(miss_by_year.index, miss_by_year.values, marker="o")
plt.title("Procenat nedostajucih vrednosti po godini")
plt.xlabel("Godina")
plt.ylabel("Rata nedostajucih vrednosti")
plt.show()

miss_by_country = dataframe.groupby("Country")["Population"].apply(lambda s: s.isna().mean()).sort_values(ascending=False)

plt.figure(figsize=(8,4))
miss_by_country.head(20).plot(kind="bar",  edgecolor="black", linewidth=1)
plt.title("Raspodela procenta nedostajućih vrednosti populacije po državama")
plt.ylabel("Procenat nedostajućih vrednosti")
plt.show()

full_missing = miss_by_country[miss_by_country == 1.0].index.tolist()
print("Broj država kojima populacija potpuno nedostaje:", len(full_missing))
print("Prvih 30 država:", full_missing[:30])

Broj država kojima populacija potpuno nedostaje: 48
Prvih 30 država: ['Antigua and Barbuda', 'Dominica', 'Barbados', 'Bahrain', 'Bahamas', 'Brunei Darussalam', 'Bolivia (Plurinational State of)', 'Gambia', 'Egypt', 'Democratic Republic of the Congo', 'Cuba', "Côte d'Ivoire", "Democratic People's Republic of Korea", 'Congo', 'Czechia', 'Cook Islands', 'The former Yugoslav republic of Macedonia', 'United States of America', 'United Republic of Tanzania', 'United Kingdom of Great Britain and Northern Ireland', 'Marshall Islands', 'Niue', 'Oman', 'Nauru', 'New Zealand', 'Micronesia (Federated States of)', 'Monaco', 'Kyrgyzstan', 'Kuwait', 'Libya']

df = dataframe.sort_values(["Country","Year"]).copy()
df["Population"] = pd.to_numeric(df["Population"])

prev = df.groupby("Country")["Population"].shift(1)
df["Population growth"] = (df["Population"] - prev) / prev

extreme = df["Population growth"].abs().sort_values(ascending=False).head(20)
print(df.loc[extreme.index, ["Country","Year","Population","Population growth"]].to_string(index=False))

               Country  Year  Population  Population growth
               Hungary  2011   9971727.0       81069.951220
              Ethiopia  2008  83184892.0       10206.987729
                  Iraq  2015  36115649.0       10121.098935
              Maldives  2015     49163.0        1198.097561
                 Benin  2005   7982225.0        1028.433196
              Cameroon  2009  19432541.0        1022.950943
               Burundi  2001   6555829.0        1011.326899
          Turkmenistan  2003   4655741.0        1008.484172
                  Peru  2010  29373646.0        1006.430325
             Nicaragua  2002   5171734.0         998.368889
Bosnia and Herzegovina  2008   3763599.0         996.244038
                  Mali  2013  16477818.0         987.649307
               Uruguay  2014   3419546.0         980.218364
              Pakistan  2005  15399667.0         974.712285
  Syrian Arab Republic  2015  18734987.0         972.802537
                Turkey  2009  71339185.0         957.447778
            Tajikistan  2007   7152385.0         945.458251
               Germany  2015  81686611.0         908.397284
                Bhutan  2009    714458.0         897.689308
                  Chad  2006   1421597.0         845.692674

g = df[df["Country"] == "Hungary"][["Year","Population"]].sort_values("Year")
print(g.to_string(index=False))

g2 = df[df["Country"] == "Hungary"][["Year","Population","Population growth"]].sort_values("Year")
print(g2.to_string(index=False))

 Year  Population
 2000    121971.0
 2001   1187576.0
 2002    115868.0
 2003   1129552.0
 2004    117146.0
 2005     18765.0
 2006     17137.0
 2007     15578.0
 2008    138188.0
 2009     12265.0
 2010       123.0
 2011   9971727.0
 2012    992362.0
 2013    989382.0
 2014   9866468.0
 2015    984328.0
 Year  Population  Population growth
 2000    121971.0                NaN
 2001   1187576.0           8.736544
 2002    115868.0          -0.902433
 2003   1129552.0           8.748610
 2004    117146.0          -0.896290
 2005     18765.0          -0.839815
 2006     17137.0          -0.086757
 2007     15578.0          -0.090973
 2008    138188.0           7.870715
 2009     12265.0          -0.911244
 2010       123.0          -0.989971
 2011   9971727.0       81069.951220
 2012    992362.0          -0.900482
 2013    989382.0          -0.003003
 2014   9866468.0           8.972354
 2015    984328.0          -0.900235

df = dataframe.sort_values(["Country","Year"]).copy()
df["Population"] = pd.to_numeric(df["Population"], errors="coerce")

c = "Hungary"
g = df[df["Country"]==c][["Year","Population"]].sort_values("Year")

plt.figure(figsize=(9,4))
plt.plot(g["Year"], g["Population"], marker="o")
plt.title("Populacija Hungary")
plt.xlabel("Godina")
plt.ylabel("Populacija")
plt.grid(True, alpha=0.3)
plt.show()

for state in ["Luxembourg","Maldives","Germany","India"]:
    g = dataframe[dataframe["Country"] == state].sort_values("Year")

    y = g["Population"]
    plt.figure(figsize=(8,3))
    plt.plot(g["Year"], y, marker="o")
    plt.title(f"{state} — Populacija kroz vreme")
    plt.xlabel("Godina")
    plt.ylabel("Populaicja")
    plt.grid(True, alpha=0.3)
    plt.show()

df = dataframe.sort_values(["Country","Year"]).copy()
df["Population"] = pd.to_numeric(df["Population"])
prev = df.groupby("Country")["Population"].shift(1)
df["pop_growth"] = (df["Population"] - prev) / prev

top = df[df["pop_growth"].notna()].copy()
top["abs_growth"] = top["pop_growth"].abs()

print(top.sort_values("abs_growth", ascending=False).head(20)[["Country","Year","Population","pop_growth"]].to_string(index=False))

               Country  Year  Population   pop_growth
               Hungary  2011   9971727.0 81069.951220
              Ethiopia  2008  83184892.0 10206.987729
                  Iraq  2015  36115649.0 10121.098935
              Maldives  2015     49163.0  1198.097561
                 Benin  2005   7982225.0  1028.433196
              Cameroon  2009  19432541.0  1022.950943
               Burundi  2001   6555829.0  1011.326899
          Turkmenistan  2003   4655741.0  1008.484172
                  Peru  2010  29373646.0  1006.430325
             Nicaragua  2002   5171734.0   998.368889
Bosnia and Herzegovina  2008   3763599.0   996.244038
                  Mali  2013  16477818.0   987.649307
               Uruguay  2014   3419546.0   980.218364
              Pakistan  2005  15399667.0   974.712285
  Syrian Arab Republic  2015  18734987.0   972.802537
                Turkey  2009  71339185.0   957.447778
            Tajikistan  2007   7152385.0   945.458251
               Germany  2015  81686611.0   908.397284
                Bhutan  2009    714458.0   897.689308
                  Chad  2006   1421597.0   845.692674

df = dataframe.sort_values(["Country","Year"]).copy()
df["Population"] = pd.to_numeric(df["Population"])
df.loc[df["Population"] <= 0, "Population"] = np.nan

prev = df.groupby("Country")["Population"].shift(1)
ratio = df["Population"] / prev

spike = df["Population"].notna() & prev.notna() & ((ratio < 0.7) | (ratio > 1.3))

bad_countries = df.loc[spike, "Country"].unique().tolist()
print("Broj država sa spike-ovima:", len(bad_countries))
print("Primer:", bad_countries[:30])

Broj država sa spike-ovima: 143
Primer: ['Afghanistan', 'Albania', 'Algeria', 'Angola', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bangladesh', 'Belarus', 'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bosnia and Herzegovina', 'Botswana', 'Brazil', 'Bulgaria', 'Burkina Faso', 'Burundi', 'Cabo Verde', 'Cambodia', 'Cameroon', 'Canada', 'Central African Republic', 'Chad', 'Chile', 'China', 'Colombia']

dataframe.loc[dataframe["Country"].isin(bad_countries), "Population"] = np.nan

pop = pd.to_numeric(dataframe["Population"])

missing_count = pop.isna().sum()
missing_percent = pop.isna().mean() * 100

print("Nedostajuci redovi:", missing_count)
print("Nedostajuci % feature-a Population", round(missing_percent,2), "%")

Nedostajuci redovi: 2936
Nedostajuci % feature-a Population 99.93 %

url = "https://api.worldbank.org/v2/country/all/indicator/SP.POP.TOTL?date=2000:2015&format=json&per_page=20000"

r = requests.get(url)
data = r.json()[1]

pop = pd.DataFrame([{
    "Country": d["country"]["value"],
    "Year": int(d["date"]),
    "PopulationWB": d["value"]
} for d in data if d["value"] is not None])

print(pop.head())

                       Country  Year  PopulationWB
0  Africa Eastern and Southern  2015     607123269
1  Africa Eastern and Southern  2014     590968990
2  Africa Eastern and Southern  2013     575202699
3  Africa Eastern and Southern  2012     559609961
4  Africa Eastern and Southern  2011     544737983

your = set(dataframe["Country"].unique())
worldBankData = set(pop["Country"].unique())
print(your - worldBankData)

{'Iran (Islamic Republic of)', 'The former Yugoslav republic of Macedonia', 'Kyrgyzstan', 'Venezuela (Bolivarian Republic of)', "Lao People's Democratic Republic", 'Saint Vincent and the Grenadines', 'United States of America', 'Saint Lucia', 'Congo', 'Saint Kitts and Nevis', 'Bahamas', 'Cook Islands', 'Yemen', 'Niue', 'Republic of Moldova', 'Turkey', 'Slovakia', "Democratic People's Republic of Korea", 'Republic of Korea', 'Gambia', 'Swaziland', 'Bolivia (Plurinational State of)', 'Democratic Republic of the Congo', 'Somalia', 'United Republic of Tanzania', 'Micronesia (Federated States of)', "Côte d'Ivoire", 'United Kingdom of Great Britain and Northern Ireland', 'Egypt'}

name_map = {
    "Bahamas": "Bahamas, The",
    "Bolivia (Plurinational State of)": "Bolivia",
    "Côte d'Ivoire": "Cote d'Ivoire",
    "Congo": "Congo, Rep.",
    "Democratic Republic of the Congo": "Congo, Dem. Rep.",
    "Democratic People's Republic of Korea": "Korea, Dem. People's Rep.",
    "Egypt": "Egypt, Arab Rep.",
    "Iran (Islamic Republic of)": "Iran, Islamic Rep.",
    "Gambia": "Gambia, The",
    "Kyrgyzstan": "Kyrgyz Republic",
    "Lao People's Democratic Republic": "Lao PDR",
    "United Republic of Tanzania": "Tanzania",
    "Micronesia (Federated States of)": "Micronesia, Fed. Sts.",
    "Republic of Korea": "Korea, Rep.",
    "Republic of Moldova": "Moldova",
    "Saint Vincent and the Grenadines": "St. Vincent and the Grenadines",
    "Saint Lucia": "St. Lucia",
    "Slovakia": "Slovak Republic",
    "Venezuela (Bolivarian Republic of)": "Venezuela, RB",
    "United States of America": "United States",
    "The former Yugoslav republic of Macedonia": "North Macedonia",
    "United Kingdom of Great Britain and Northern Ireland": "United Kingdom",
    "Yemen": "Yemen, Rep.",
    "Saint Kitts and Nevis": "St. Kitts and Nevis",
    "Swaziland": "Eswatini",
    "Turkey": "Turkiye"
}

df = dataframe.copy()
df["Year"] = pd.to_numeric(df["Year"]).astype(int)
df["Country_wb"] = df["Country"].str.strip().replace(name_map)
copyDataframe = pop.copy()
copyDataframe["Year"] = copyDataframe["Year"].astype(int)

merged = df.merge(
    copyDataframe.rename(columns={"Country": "Country_wb"}),
    on=["Country_wb", "Year"],
    how="left"
)

merged["Population"] = pd.to_numeric(merged["Population"])
merged["Population"] = merged["Population"].fillna(merged["PopulationWB"])

merged.loc[merged["Population"] < 10_000, "Population"] = merged["PopulationWB"]

merged.drop(columns=["PopulationWB"], inplace=True)
dataframe = merged

print("Remaining missing Population:", dataframe["Population"].isna().sum())

Remaining missing Population: 18

print(pop[pop["Country"]=="Somalia"][["Country","Year","PopulationWB"]].sort_values("Year").to_string(index=False))

Empty DataFrame
Columns: [Country, Year, PopulationWB]
Index: []

all_missing = (
    dataframe.groupby("Country")["Population"]
    .apply(lambda s: s.isna().mean())
    .loc[lambda x: x == 1.0]
    .index
)

print("Drzave sa 100% nedostajucom populacijom:", len(all_missing))
print(all_missing.tolist())

Drzave sa 100% nedostajucom populacijom: 3
['Cook Islands', 'Niue', 'Somalia']

dataframe = dataframe[~dataframe["Country"].isin(all_missing)].copy()

dataframe["GDP"] = pd.to_numeric(dataframe["GDP"])
dataframe.loc[dataframe["GDP"] < 0, "GDP"] = np.nan
dataframe["GDP"].describe()

count      2487.000000
mean       7491.826364
std       14276.591042
min           1.681350
25%         464.674822
50%        1771.586620
75%        5925.549329
max      119172.741800
Name: GDP, dtype: float64

dataframe["GDP_diff"] = dataframe.groupby("Country")["GDP"].diff().abs()
dataframe.sort_values("GDP_diff", ascending=False)[["Country","Year","GDP","GDP_diff"]].head(20)

gdp_suspicious = dataframe[(dataframe['Status'] == 'Developed') & (dataframe['GDP'] < 500)][['Country','Year','GDP']]
print(gdp_suspicious.to_string())

          Country  Year         GDP
125     Australia  2002  281.817630
137       Austria  2006  443.993610
396      Bulgaria  2003  271.468240
397      Bulgaria  2002  287.534843
399      Bulgaria  2000  169.285860
1174      Iceland  2011   46.217000
1282        Italy  2015  349.147550
1289        Italy  2008  464.184650
1296        Italy  2001   24.819000
1297        Italy  2000  251.242600
1536    Lithuania  2001  353.147337
1851  New Zealand  2009  282.941930
2046       Poland  2008  141.446880
2048       Poland  2006   94.772600
2120      Romania  2014   12.277330
2123      Romania  2011   92.277825
2346     Slovenia  2014  242.672860
2426        Spain  2014  296.472250

countries = ["Luxembourg","Norway","Qatar","Belgium","New Zealand"]
tmp = dataframe[dataframe["Country"].isin(countries)].sort_values(["Country","Year"])

for country in countries:
    g = tmp[tmp["Country"] == country]

    plt.figure(figsize=(7,3))
    plt.plot(g["Year"], g["GDP"], marker="o")
    plt.title(country)
    plt.xlabel("Godina")
    plt.ylabel("GDP")
    plt.grid(True, alpha=0.3)
    plt.show()

miss_year = dataframe.groupby("Year")["GDP"].apply(lambda x: x.isna().mean())

plt.figure(figsize=(7,3))
plt.plot(miss_year.index, miss_year*100, marker="o")
plt.title("GDP missing po godini (%)")
plt.xlabel("Godina")
plt.ylabel("Missing (%)")
plt.grid(True, alpha=0.3)
plt.show()

miss_country = dataframe.groupby("Country")["GDP"].apply(lambda x: x.isna().mean()).sort_values(ascending=False)

top20 = miss_country.head(20).sort_values()

plt.figure(figsize=(8,6))
plt.barh(top20.index, top20.values * 100)
plt.title("Procenat nedostajucih vrednosti")
plt.xlabel("Missing (%)")
plt.ylabel("Drzave")
plt.grid(True, axis="x", alpha=0.3)
plt.show()

miss_status = dataframe.groupby("Status")["GDP"].apply(lambda s: s.isna().mean())

plt.figure()
plt.bar(miss_status.index.astype(str), miss_status.values*100)
plt.title("GDP missingness by Status (%)")
plt.xlabel("Status")
plt.ylabel("Missing %")
plt.show()

m = dataframe["GDP"].isna()
tab = pd.crosstab(dataframe["Status"], m)

chi2, p_chi, dof, exp = stats.chi2_contingency(tab)
print("Chi-square p-value:", p_chi)

Chi-square p-value: 0.11774689286825281

y = "Life expectancy"
m = dataframe["GDP"].isna()

a = dataframe.loc[m, y].dropna()
b = dataframe.loc[~m, y].dropna()

print("p =", stats.mannwhitneyu(a, b, alternative="two-sided").pvalue)

p = 0.07325169718400026

missing_gdp = dataframe["GDP"].isna()

numeric_cols = dataframe.select_dtypes(include=[np.number]).columns
numeric_cols = [col for col in numeric_cols if col != "GDP"]

rows = []

for col in numeric_cols:
    group_missing = dataframe.loc[missing_gdp, col].dropna()
    group_present = dataframe.loc[~missing_gdp, col].dropna()

    if len(group_missing) < 10 or len(group_present) < 10:
        continue

    p = stats.mannwhitneyu(group_missing, group_present, alternative="two-sided").pvalue

    rows.append({
        "feature": col,
        "pvalue": p,
        "mean_GDP_missing": group_missing.mean(),
        "mean_GDP_present": group_present.mean(),
        "n_missing": len(group_missing),
        "n_present": len(group_present),
    })

result = pd.DataFrame(rows).sort_values("pvalue")
result.head(20)

pairs = [
    ("Schooling", "GDP"),
    ("Income composition of resources", "GDP")
]

for x,y in pairs:
    tmp = dataframe[[x,y]].dropna()
    plt.figure(figsize=(5,4))
    plt.scatter(tmp[x], tmp[y], alpha=0.25, s=10)
    plt.title(f"{y} vs {x}")
    plt.xlabel(x)
    plt.ylabel(y)
    plt.show()

cols = ["GDP","Schooling","Income composition of resources","Life expectancy","Adult Mortality","Total expenditure","Alcohol"]
print(dataframe[cols].corr(numeric_only=True)["GDP"].sort_values(ascending=False))

GDP                                1.000000
Life expectancy                    0.461250
Income composition of resources    0.460341
Schooling                          0.448273
Alcohol                            0.354455
Total expenditure                  0.138364
Adult Mortality                   -0.295616
Name: GDP, dtype: float64

gdp = dataframe["GDP"].copy()

country_med = dataframe.groupby("Country")["GDP"].transform("median")

collapsed = gdp.notna() & country_med.notna() & (gdp < 0.2 * country_med)

dataframe.loc[collapsed, "GDP"] = np.nan

df = dataframe.sort_values(["Country","Year"]).copy()

prev = df.groupby("Country")["GDP"].shift(1)
next_ = df.groupby("Country")["GDP"].shift(-1)

bad_prev = prev.notna() & ((df["GDP"] / prev < 0.2) | (df["GDP"] / prev > 5))
bad_next = next_.notna() & ((df["GDP"] / next_ < 0.2) | (df["GDP"] / next_ > 5))

bad_jump = df["GDP"].notna() & (bad_prev | bad_next)

dataframe.loc[bad_jump, "GDP"] = np.nan

dataframe = dataframe.sort_values(["Country","Year"])
dataframe["GDP"] = dataframe.groupby("Country")["GDP"].transform(
    lambda s: s.interpolate(limit_direction="both")
)

work = dataframe.copy()

work["Status_encoded"] = work["Status"].map({"Developing": 0, "Developed": 1})

knn_features = [
    "GDP",
    "Schooling",
    "Income composition of resources",
    "Adult Mortality",
    "Total expenditure",
    "Alcohol",
    "Status_encoded"
]

work[knn_features] = work[knn_features].apply(pd.to_numeric)

knn_imputer = KNNImputer(n_neighbors=5, weights="distance")
work[knn_features] = knn_imputer.fit_transform(work[knn_features])

dataframe["GDP"] = work["GDP"]

dataframe = dataframe.drop(columns=["GDP_diff"], errors="ignore")

dataframe['Hepatitis B'] = pd.to_numeric(dataframe['Hepatitis B'])

dataframe['Hepatitis B'].describe()

count    2380.000000
mean       80.974790
std        25.053021
min         1.000000
25%        77.000000
50%        92.000000
75%        97.000000
max        99.000000
Name: Hepatitis B, dtype: float64

tmp = dataframe[["Polio", "Hepatitis B"]].dropna()

plt.figure(figsize=(5,4))
plt.scatter(tmp["Polio"], tmp["Hepatitis B"], s=10, alpha=0.25)
plt.title("HepB vs Polio")
plt.xlabel("Polio (%)")
plt.ylabel("Hepatitis B (%)")
plt.grid(True, alpha=0.3)
plt.show()

tmp = dataframe[["Diphtheria", "Hepatitis B"]].dropna()

plt.figure(figsize=(5,4))
plt.scatter(tmp["Diphtheria"], tmp["Hepatitis B"], s=10, alpha=0.25)
plt.title("HepB vs Diphtheria")
plt.xlabel("Diphtheria (%)")
plt.ylabel("Hepatitis B (%)")
plt.grid(True, alpha=0.3)
plt.show()

miss_by_year = (
    dataframe.groupby("Year")["Hepatitis B"]
    .apply(lambda s: s.isna().mean())
    .sort_index()
)

plt.figure(figsize=(6, 4))
plt.plot(miss_by_year.index, miss_by_year.values * 100)
plt.title("Nedostajuce vrednosti za Hepatitis B po godini (%)")
plt.xlabel("Year")
plt.ylabel("Missing (%)")
plt.grid(True, alpha=0.3)
plt.show()

miss_by_status = (
    dataframe.groupby("Status")["Hepatitis B"]
    .apply(lambda s: s.isna().mean())
)

plt.figure(figsize=(6,4))
plt.bar(miss_by_status.index.astype(str), miss_by_status.values * 100)
plt.title("Hepatitis B missingness po Statusu (%)")
plt.xlabel("Status")
plt.ylabel("Missing (%)")
plt.grid(axis="y", alpha=0.3)
plt.show()

miss_by_country = (
    dataframe.groupby("Country")["Hepatitis B"]
    .apply(lambda s: s.isna().mean())
    .sort_values(ascending=False)
)

top20 = miss_by_country.head(20).sort_values()

plt.figure(figsize=(10, 6))
plt.barh(top20.index.astype(str), top20.values * 100)
plt.title("Top 20 countries by Hepatitis B missingness (%)")
plt.xlabel("Missing (%)")
plt.ylabel("Country")
plt.grid(axis="x", alpha=0.3)
plt.tight_layout()
plt.show()

weird = ["Denmark","Norway","Iceland","Finland","Switzerland","Japan"]
dataframe[dataframe["Country"].isin(weird)].isna().mean().sort_values(ascending=False).head(10)

Hepatitis B          1.000000
Total expenditure    0.062500
Alcohol              0.052083
Status               0.000000
Life expectancy      0.000000
Year                 0.000000
Country              0.000000
infant deaths        0.000000
Adult Mortality      0.000000
Measles              0.000000
dtype: float64

m = dataframe["Hepatitis B"].isna()

plt.figure(figsize=(6,4))
plt.hist(dataframe.loc[~m, "Hepatitis B"].dropna(), bins=20, edgecolor='black')
plt.title("Distribucija Hepatitis B")
plt.xlabel("Hepatitis B")
plt.ylabel("Count")
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

tab = pd.crosstab(dataframe["Status"], m)
print((tab.div(tab.sum(axis=1), axis=0)*100).round(2))
print(stats.chi2_contingency(tab)[:2])

Hepatitis B  False  True 
Status                   
Developed    66.21  33.79
Developing   84.76  15.24
(np.float64(95.14342955568858), np.float64(1.7707895774911682e-22))

a = dataframe.loc[m, "Life expectancy"].dropna()
b = dataframe.loc[~m, "Life expectancy"].dropna()

print(stats.mannwhitneyu(a, b, alternative="two-sided"))

MannwhitneyuResult(statistic=np.float64(565124.0), pvalue=np.float64(1.943676481510776e-05))

col = "Hepatitis B"
pol = "Polio"
diph = "Diphtheria"

flag = col + "_was_missing"
dataframe[flag] = dataframe[col].isna().astype(int)

full_missing_countries = (
    dataframe.groupby("Country")[flag]
    .apply(lambda s: (s == 1).all())
)
full_missing_countries = full_missing_countries[full_missing_countries].index.tolist()

mask_full_missing = dataframe["Country"].isin(full_missing_countries) & dataframe[col].isna()
proxy_vax = dataframe[[pol, diph]].mean(axis=1, skipna=True)
dataframe.loc[mask_full_missing, col] = proxy_vax.loc[mask_full_missing]

dataframe[col] = dataframe[col].fillna(dataframe.groupby("Country")[col].transform("median"))
dataframe[col] = dataframe[col].fillna(dataframe.groupby("Status")[col].transform("median"))
dataframe[col] = dataframe[col].fillna(dataframe[col].median())

print("Broj full-missing država:", len(full_missing_countries))
print(full_missing_countries[:20])

Broj full-missing država: 9
['Denmark', 'Finland', 'Hungary', 'Iceland', 'Japan', 'Norway', 'Slovenia', 'Switzerland', 'United Kingdom of Great Britain and Northern Ireland']

cols = ["Total expenditure", "Alcohol", "Income composition of resources", "Schooling"]
ic = "Income composition of resources"

for c in cols:
    dataframe[c] = pd.to_numeric(dataframe[c])

desc = dataframe[cols].describe(percentiles=[.01,.05,.25,.5,.75,.95,.99]).T
missing_pct = (dataframe[cols].isna().mean() * 100).round(3)

print(desc)

                                  count       mean       std   min      1%  \
Total expenditure                2710.0   5.938594  2.498713  0.37  1.2309   
Alcohol                          2727.0   4.631492  4.048715  0.01  0.0100   
Income composition of resources  2771.0   0.627551  0.210904  0.00  0.0000   
Schooling                        2775.0  11.992793  3.358920  0.00  2.0720   

                                    5%     25%     50%      75%     95%  \
Total expenditure                1.930   4.260   5.755   7.4975   9.760   
Alcohol                          0.010   0.935   3.790   7.7450  11.974   
Income composition of resources  0.277   0.493   0.677   0.7790   0.892   
Schooling                        5.800  10.100  12.300  14.3000  16.800   

                                     99%     max  
Total expenditure                12.9274  17.600  
Alcohol                          13.4848  17.870  
Income composition of resources   0.9233   0.948  
Schooling                        19.0000  20.700

n = len(cols)
fig, axes = plt.subplots(
    nrows=2, ncols=n,
    figsize=(4.5*n, 7),
    gridspec_kw={"height_ratios": [3, 1]}
)

for i, col in enumerate(cols):
    x = dataframe[col].dropna()

    ax_h = axes[0, i]
    ax_h.hist(x, bins=40, edgecolor="black", linewidth=0.8)
    ax_h.set_title(col)
    ax_h.set_xlabel(col)
    ax_h.set_ylabel("Count")
    ax_h.grid(True, alpha=0.25)

    ax_b = axes[1, i]
    ax_b.boxplot(x, vert=False)
    ax_b.set_xlabel(col)
    ax_b.grid(True, alpha=0.25)

plt.tight_layout()
plt.show()

c = "Total expenditure"
s = dataframe[c].dropna()

s.describe(percentiles=[.01,.05,.25,.5,.75,.95,.99])

count    2710.000000
mean        5.938594
std         2.498713
min         0.370000
1%          1.230900
5%          1.930000
25%         4.260000
50%         5.755000
75%         7.497500
95%         9.760000
99%        12.927400
max        17.600000
Name: Total expenditure, dtype: float64

counts = dataframe.groupby("Year")["Alcohol"].apply(lambda s: (s == 0.01).sum()).sort_index()

plt.figure(figsize=(8,3))
plt.plot(counts.index, counts.values, marker="o")
plt.title("Broj slučajeva gde je Alcohol == 0.01 po godini")
plt.xlabel("Godina")
plt.ylabel("Broj")
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

ic = "Income composition of resources"
counts = dataframe.groupby("Year")[ic].apply(lambda s: (s == 0.0).sum()).sort_index()

plt.figure(figsize=(8,3))
plt.plot(counts.index, counts.values, marker="o")
plt.title("Broj slučajeva gde je Income composition == 0.0 po godini")
plt.xlabel("Godina")
plt.ylabel("Broj")
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

counts = dataframe.groupby("Year")["Schooling"].apply(lambda s: (s == 0.0).sum()).sort_index()

plt.figure(figsize=(8,3))
plt.plot(counts.index, counts.values, marker="o")
plt.title("Broj slučajeva gde je Schooling == 0.0 po godini")
plt.xlabel("Godina")
plt.ylabel("Broj")
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

df_clean = dataframe.copy()

col_alcohol = "Alcohol"
col_income = "Income composition of resources"
col_school = "Schooling"

cols = [col_alcohol, col_income, col_school]

df_clean.loc[df_clean[col_alcohol] == 0.01, col_alcohol] = np.nan
df_clean.loc[df_clean[col_income]  == 0.0,  col_income]  = np.nan
df_clean.loc[df_clean[col_school]  == 0.0,  col_school]  = np.nan

df_clean = df_clean.sort_values(["Country", "Year"])

for c in cols:
    df_clean[c] = df_clean.groupby("Country")[c].transform(
        lambda s: s.fillna(s.median()) if s.notna().any() else s
    )

    df_clean[c] = df_clean[c].fillna(df_clean[c].median())

col = "Total expenditure"

df_clean[col] = df_clean.groupby("Country")[col].transform(
    lambda s: s.fillna(s.median()) if s.notna().any() else s
)
df_clean[col] = df_clean[col].fillna(df_clean[col].median())

dataframe = df_clean

cols = ["BMI","thinness 10-19 years","thinness 5-9 years",
        "Diphtheria","Polio","Adult Mortality","Life expectancy"]

for c in cols:
    dataframe[c] = pd.to_numeric(dataframe[c])

kolone = ["BMI", "thinness 10-19 years", "thinness 5-9 years", "Diphtheria", "Polio"]

for kol in kolone:
    df = dataframe[["Country", "Year", kol]].dropna().sort_values(["Country", "Year"]).copy()

    df["skok_po_godini"] = df.groupby("Country")[kol].diff().abs()

    prag = df["skok_po_godini"].quantile(0.99)

    top = df[df["skok_po_godini"] > prag].sort_values("skok_po_godini", ascending=False).head(10)

    print(f"\n{kol}:")
    print(f"Prag (99. percentil skoka): {prag:.3f}")
    print(top[["Country", "Year", kol, "skok_po_godini"]].to_string(index=False))

BMI:
Prag (99. percentil skoka): 54.600
             Country  Year  BMI  skok_po_godini
            Kiribati  2004 71.4            63.8
               Tonga  2008 71.5            63.7
              Kuwait  2015 71.4            63.6
               Samoa  2008 71.4            63.5
               Samoa  2006  7.3            62.4
               Tonga  2006  7.1            62.3
              Kuwait  2013  7.2            62.3
            Kiribati  2003  7.6            62.1
United Arab Emirates  2014 62.4            55.9
             Tunisia  2015 61.2            55.0

thinness 10-19 years:
Prag (99. percentil skoka): 8.800
     Country  Year  thinness 10-19 years  skok_po_godini
    Pakistan  2007                   2.8            18.2
    Pakistan  2012                  19.8            17.8
 Afghanistan  2002                  19.9            17.8
  Bangladesh  2005                  19.9            17.8
South Africa  2006                   1.6            10.0
     Namibia  2009                   1.9             9.6
    Botswana  2003                   1.9             9.5
     Lesotho  2002                   1.6             9.5
    Zimbabwe  2001                   1.6             9.4
       Niger  2010                   1.7             9.3

thinness 5-9 years:
Prag (99. percentil skoka): 8.800
     Country  Year  thinness 5-9 years  skok_po_godini
  Bangladesh  2003                 2.9            18.2
    Pakistan  2009                 2.9            18.2
    Pakistan  2014                19.8            17.8
  Bangladesh  2008                19.9            17.8
 Afghanistan  2003                19.9            17.7
South Africa  2008                 1.7            10.0
    Botswana  2003                 1.8             9.5
     Lesotho  2002                 1.6             9.5
     Namibia  2009                 1.9             9.5
    Zimbabwe  2001                 1.7             9.5

Diphtheria:
Prag (99. percentil skoka): 83.000
        Country  Year  Diphtheria  skok_po_godini
        Belarus  2003         5.0            94.0
        Belarus  2004        99.0            94.0
    Saint Lucia  2001        99.0            92.0
     Cabo Verde  2011         9.0            90.0
Solomon Islands  2007         9.0            90.0
Solomon Islands  2011        99.0            90.0
           Peru  2001         9.0            89.0
          Ghana  2014        98.0            89.0
      Swaziland  2015         9.0            89.0
        Ukraine  2008         9.0            89.0

Polio:
Prag (99. percentil skoka): 84.000
        Country  Year  Polio  skok_po_godini
    Saint Lucia  2001   99.0            92.0
        Comoros  2002   98.0            91.0
    Saint Lucia  2002    9.0            90.0
        Comoros  2003    8.0            90.0
Solomon Islands  2006   99.0            90.0
     Cabo Verde  2011    9.0            90.0
        Belarus  2008   98.0            89.0
          Kenya  2011   97.0            88.0
        Belarus  2007    9.0            88.0
        Ecuador  2004    9.0            88.0

spike_countries = ["Kiribati", "Tonga", "Kuwait", "Tunisia"]

tmp = dataframe.loc[dataframe["Country"].isin(spike_countries), ["Country", "Year", "BMI"]].copy()
tmp["BMI"] = pd.to_numeric(tmp["BMI"], errors="coerce")

for country in spike_countries:
    s = tmp[tmp["Country"] == country].dropna(subset=["BMI"]).sort_values("Year")

    plt.figure(figsize=(8,3))
    plt.plot(s["Year"], s["BMI"], marker="o", linewidth=1.2)
    plt.title(country)
    plt.xlabel("Godina")
    plt.ylabel("BMI")
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()

dataframe = dataframe.drop(columns=["BMI"], errors="ignore")

dataframe = dataframe.drop(columns=["thinness 5-9 years"], errors="ignore")

for c in ["Polio", "Diphtheria"]:
    dataframe[c] = pd.to_numeric(dataframe[c])
    dataframe[c] = dataframe[c].clip(0, 100)

cols = ["Polio", "Diphtheria"]


for c in cols:
    x = dataframe[c].dropna()

    plt.figure(figsize=(7,4))
    plt.hist(x, bins=40, edgecolor="black", linewidth=0.8)
    plt.title(f"Distribucija: {c}")
    plt.xlabel(c)
    plt.ylabel("Broj")
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()

kolone = ["Polio", "Diphtheria"]

for kol in kolone:
    x = dataframe[kol]

    ukupno = x.notna().sum()
    ispod_10 = (x < 10).sum()
    nula = (x == 0).sum()

    print(f"\n{kol}:")
    print(f"Opseg vrednosti: {x.min():.1f} – {x.max():.1f}")
    print(f"Vrednosti < 10:  {ispod_10}/{ukupno} ({ispod_10/ukupno*100:.2f}%)")
    print(f"Vrednosti = 0:   {nula}")

Polio:
Opseg vrednosti: 3.0 – 99.0
Vrednosti < 10:  167/2901 (5.76%)
Vrednosti = 0:   0

Diphtheria:
Opseg vrednosti: 2.0 – 99.0
Vrednosti < 10:  166/2901 (5.72%)
Vrednosti = 0:   0

cols_show = ["Country", "Year", "Polio", "Diphtheria", "Status"]

low_polio = dataframe.loc[
    dataframe["Polio"] < 10,
    cols_show
].copy()
low_polio["Polio"] = low_polio["Polio"]

print(low_polio.sort_values(["Polio", "Country", "Year"]).head(40).to_string(index=False))

                         Country  Year  Polio  Diphtheria     Status
                          Angola  2000    3.0        28.0 Developing
                            Chad  2000    3.0        36.0 Developing
                            Chad  2008    3.0        19.0 Developing
Democratic Republic of the Congo  2001    3.0         3.0 Developing
               Equatorial Guinea  2012    3.0        24.0 Developing
               Equatorial Guinea  2013    3.0         3.0 Developing
                          Angola  2003    4.0         4.0 Developing
                          Angola  2004    4.0         4.0 Developing
        Central African Republic  2001    4.0         4.0 Developing
                            Chad  2011    4.0        33.0 Developing
Democratic Republic of the Congo  2002    4.0        38.0 Developing
                           Niger  2011    4.0        75.0 Developing
                         Nigeria  2002    4.0        25.0 Developing
                     Afghanistan  2004    5.0         5.0 Developing
                           Congo  2003    5.0         5.0 Developing
               Equatorial Guinea  2005    5.0        39.0 Developing
                           Haiti  2000    5.0        41.0 Developing
Lao People's Democratic Republic  2005    5.0        49.0 Developing
                     South Sudan  2013    5.0        45.0 Developing
            Syrian Arab Republic  2013    5.0        41.0 Developing
            Syrian Arab Republic  2015    5.0        41.0 Developing
                     Afghanistan  2015    6.0        65.0 Developing
Democratic Republic of the Congo  2005    6.0         6.0 Developing
                          Guinea  2009    6.0        57.0 Developing
                           Haiti  2005    6.0         6.0 Developing
Lao People's Democratic Republic  2008    6.0        61.0 Developing
                      Madagascar  2001    6.0         6.0 Developing
                         Nigeria  2008    6.0        53.0 Developing
                           Samoa  2011    6.0        65.0 Developing
                         Senegal  2002    6.0         6.0 Developing
                           Sudan  2002    6.0         6.0 Developing
            Syrian Arab Republic  2011    6.0        72.0 Developing
                          Angola  2015    7.0        64.0 Developing
                         Comoros  2000    7.0         7.0 Developing
                         Comoros  2001    7.0         7.0 Developing
                   Côte d'Ivoire  2001    7.0        66.0 Developing
                   Côte d'Ivoire  2002    7.0        64.0 Developing
                        Ethiopia  2011    7.0        65.0 Developing
                        Ethiopia  2012    7.0        69.0 Developing
                        Ethiopia  2013    7.0        72.0 Developing

low_diph = dataframe.loc[
    dataframe["Diphtheria"]< 10,
    cols_show
].copy()
low_diph["Diphtheria"] = low_diph["Diphtheria"]

print(low_diph.sort_values(["Diphtheria", "Country", "Year"]).head(40).to_string(index=False))

                           Country  Year  Polio  Diphtheria     Status
                 Equatorial Guinea  2014   24.0         2.0 Developing
  Democratic Republic of the Congo  2001    3.0         3.0 Developing
                 Equatorial Guinea  2013    3.0         3.0 Developing
                          Ethiopia  2000   55.0         3.0 Developing
                            Angola  2003    4.0         4.0 Developing
                            Angola  2004    4.0         4.0 Developing
          Central African Republic  2001    4.0         4.0 Developing
                              Chad  2006   49.0         4.0 Developing
                              Chad  2012   51.0         4.0 Developing
  Democratic Republic of the Congo  2000   42.0         4.0 Developing
                 Equatorial Guinea  2006   52.0         4.0 Developing
                          Ethiopia  2004   54.0         4.0 Developing
                           Nigeria  2006   46.0         4.0 Developing
                       Afghanistan  2004    5.0         5.0 Developing
                           Belarus  2003   53.0         5.0 Developing
                             Congo  2003    5.0         5.0 Developing
                          Ethiopia  2007   61.0         5.0 Developing
                            Guinea  2001   52.0         5.0 Developing
  Lao People's Democratic Republic  2007   46.0         5.0 Developing
                           Liberia  2014   49.0         5.0 Developing
                              Togo  2001   51.0         5.0 Developing
                           Ukraine  2011   54.0         5.0 Developing
Venezuela (Bolivarian Republic of)  2008   76.0         5.0 Developing
                            Angola  2009   63.0         6.0 Developing
                          Cambodia  2001   59.0         6.0 Developing
  Democratic Republic of the Congo  2005    6.0         6.0 Developing
  Democratic Republic of the Congo  2010   76.0         6.0 Developing
                            Guinea  2004   65.0         6.0 Developing
                            Guinea  2008   59.0         6.0 Developing
                     Guinea-Bissau  2003   65.0         6.0 Developing
                             Haiti  2005    6.0         6.0 Developing
                             Haiti  2006   61.0         6.0 Developing
                             Haiti  2015   56.0         6.0 Developing
                           Liberia  2005   66.0         6.0 Developing
                           Liberia  2006   66.0         6.0 Developing
                        Madagascar  2001    6.0         6.0 Developing
                       Philippines  2015   79.0         6.0 Developing
                           Senegal  2002    6.0         6.0 Developing
                             Sudan  2002    6.0         6.0 Developing
                             Benin  2005   73.0         7.0 Developing

df2 = dataframe.sort_values(["Country", "Year"]).copy()

for col in ["Polio", "Diphtheria"]:
    df2[col] = pd.to_numeric(df2[col])

    df2[col] = df2.groupby("Country")[col].transform(lambda s: s.interpolate(limit_direction="both"))

    df2[col] = df2.groupby("Country")[col].transform(
        lambda s: s.fillna(s.median()) if s.notna().any() else s
    )

    df2[col] = df2[col].fillna(df2[col].median())
    
dataframe = df2

col = "thinness 10-19 years"
col = "thinness 10-19 years"
dataframe[col] = pd.to_numeric(dataframe[col])

miss = dataframe[col].isna().mean() * 100
desc = dataframe[col].describe()
print(f"Nedostajuce vrednosti % za '{col}': {miss:.2f}%")
print(desc)

plt.figure(figsize=(7,4))
plt.hist(dataframe[col].dropna(), bins=40, edgecolor="black", linewidth=0.8)
plt.title("Distribucija: thinness 10–19 years")
plt.xlabel(col)
plt.ylabel("Broj")
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

Nedostajuce vrednosti % za 'thinness 10-19 years': 1.16%
count    2886.000000
mean        4.829522
std         4.428383
min         0.100000
25%         1.600000
50%         3.300000
75%         7.175000
max        27.700000
Name: thinness 10-19 years, dtype: float64

df_sorted = dataframe.sort_values(["Country", "Year"]).copy()
df_sorted[col] = pd.to_numeric(df_sorted[col])

tmp = df_sorted[["Country", "Year", col]].dropna().copy()
tmp["skok_po_godini"] = tmp.groupby("Country")[col].diff().abs()

prag = tmp["skok_po_godini"].quantile(0.99)
spikes = tmp[tmp["skok_po_godini"] > prag].sort_values("skok_po_godini", ascending=False)

print("Top 25 skokova:")
print(spikes.head(25).to_string(index=False))

Top 25 skokova:
                         Country  Year  thinness 10-19 years  skok_po_godini
                        Pakistan  2007                   2.8            18.2
                        Pakistan  2012                  19.8            17.8
                     Afghanistan  2002                  19.9            17.8
                      Bangladesh  2005                  19.9            17.8
                    South Africa  2006                   1.6            10.0
                         Namibia  2009                   1.9             9.6
                        Botswana  2003                   1.9             9.5
                         Lesotho  2002                   1.6             9.5
                        Zimbabwe  2001                   1.6             9.4
                           Niger  2010                   1.7             9.3
                    Burkina Faso  2003                   1.7             9.3
                         Nigeria  2012                   1.7             9.3
Democratic Republic of the Congo  2008                   1.8             9.3
                            Mali  2001                   1.8             9.2
                         Senegal  2008                   1.8             9.2
                     Timor-Leste  2014                   1.9             9.2
                            Chad  2003                   1.9             9.2
                        Ethiopia  2011                   1.9             9.1
                       Indonesia  2003                   1.9             9.1
                        Cambodia  2014                   1.9             9.1
        Central African Republic  2004                   9.9             8.9
Democratic Republic of the Congo  2013                   9.9             8.9
Lao People's Democratic Republic  2006                   9.9             8.9
                         Eritrea  2002                   9.9             8.9
                         Senegal  2013                   9.9             8.9

for country in spikes["Country"].head(3).unique():
    s = df_sorted[df_sorted["Country"] == country][["Year", col]].dropna().sort_values("Year")

    plt.figure(figsize=(9,3))
    plt.plot(s["Year"], s[col], marker="o", linewidth=1.2)
    plt.title(f"Thinness 10–19 kroz godine: {country}")
    plt.xlabel("Godina")
    plt.ylabel(col)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()

df_thin = dataframe.sort_values(["Country", "Year"]).copy()
df_thin[col] = pd.to_numeric(df_thin[col], errors="coerce")

df_thin.loc[(df_thin[col] < 0) | (df_thin[col] > 50), col] = np.nan

df_thin[col] = df_thin.groupby("Country")[col].transform(
    lambda s: s.interpolate(limit_direction="both")
)

df_thin[col] = df_thin.groupby("Country")[col].transform(
    lambda s: s.fillna(s.median()) if s.notna().any() else s
)

df_thin[col] = df_thin[col].fillna(df_thin[col].median())

dataframe = df_thin

dataframe = dataframe.drop(columns=["Adult Mortality"], errors="ignore")

y = "Life expectancy"
dataframe[y] = pd.to_numeric(dataframe[y])

missing = dataframe[dataframe[y].isna()]
print("Missing Life expectancy rows:", len(missing))

Missing Life expectancy rows: 8

countries = missing["Country"].value_counts()

print("Države sa nedostajućim vrednostima za Life expectancy:", len(countries))
print(countries.head(10))

Države sa nedostajućim vrednostima za Life expectancy: 8
Country
Dominica                 1
Marshall Islands         1
Monaco                   1
Nauru                    1
Palau                    1
Saint Kitts and Nevis    1
San Marino               1
Tuvalu                   1
Name: count, dtype: int64

dataframe = dataframe[dataframe["Life expectancy"].notna()].copy()

num_cols = dataframe.select_dtypes(include=[np.number]).columns.tolist()
if "Year" in num_cols:
    num_cols.remove("Year")

n = len(num_cols)
cols = 4
rows = (n + cols - 1) // cols

fig, axes = plt.subplots(rows*2, cols, figsize=(cols*4.2, rows*4.0))
axes = axes.reshape(rows*2, cols)

for i, c in enumerate(num_cols):
    r, k = (i // cols)*2, i % cols
    x = pd.to_numeric(dataframe[c], errors="coerce").dropna()

    ax = axes[r, k]
    ax.hist(x, bins=40, edgecolor="black", linewidth=0.8)
    ax.set_title(f"Distribucija: {c}", fontsize=10)
    ax.grid(True, alpha=0.25)

    ax2 = axes[r+1, k]
    ax2.boxplot(x, vert=False,
                flierprops=dict(marker="o", markersize=2, markerfacecolor="none",
                                markeredgecolor="black", markeredgewidth=0.6))
    ax2.grid(True, alpha=0.25)

for j in range(i+1, rows*cols):
    fig.delaxes(axes[(j//cols)*2, j%cols])
    fig.delaxes(axes[(j//cols)*2+1, j%cols])

plt.tight_layout()
plt.show()

print("PRE:")

print((dataframe[["Alcohol","Total expenditure"]]
       .describe(percentiles=[.01,.99])).T[["min","1%","50%","99%","max"]])

PRE:
                    min      1%   50%      99%    max
Alcohol            0.02  0.0200  4.13  13.9345  17.87
Total expenditure  0.37  1.2522  5.73  12.5879  17.60

def set_outside_nan(col, lo, hi):
    if col in dataframe.columns:
        x = pd.to_numeric(dataframe[col], errors="coerce")
        before_nan = x.isna().sum()
        bad = (x < lo) | (x > hi)
        dataframe[col] = x.mask(bad, np.nan)
        after_nan = dataframe[col].isna().sum()
        print(f"{col}: van [{lo},{hi}] -> dodato NaN = {after_nan - before_nan}")

def clip_pct(col, lo=0.01, hi=0.99):
    if col in dataframe.columns:
        x = dataframe[col].dropna()
        qlo, qhi = x.quantile(lo), x.quantile(hi)

        before = dataframe[col].copy()
        dataframe[col] = dataframe[col].clip(qlo, qhi)

        changed = (before != dataframe[col]).sum()
        print(f"{col}: clip {lo:.0%}-{hi:.0%} na [{qlo:.3f}, {qhi:.3f}] -> promenjeno = {int(changed)}")

for c in ["Hepatitis B", "Polio", "Diphtheria"]:
    set_outside_nan(c, 0, 100)

set_outside_nan("Income composition of resources", 0, 1)
set_outside_nan("Schooling", 0, 25)
set_outside_nan("Alcohol", 0, 100)
set_outside_nan("Total expenditure", 0, 100)

for c in ["Alcohol", "Total expenditure"]:
    clip_pct(c, 0.01, 0.99)

Hepatitis B: van [0,100] -> dodato NaN = 0
Polio: van [0,100] -> dodato NaN = 0
Diphtheria: van [0,100] -> dodato NaN = 0
Income composition of resources: van [0,1] -> dodato NaN = 0
Schooling: van [0,25] -> dodato NaN = 0
Alcohol: van [0,100] -> dodato NaN = 0
Total expenditure: van [0,100] -> dodato NaN = 0
Alcohol: clip 1%-99% na [0.020, 13.934] -> promenjeno = 30
Total expenditure: clip 1%-99% na [1.252, 12.588] -> promenjeno = 60

print("\nPOSLE:")
print((dataframe[["Alcohol","Total expenditure"]]
       .describe(percentiles=[.01,.99])).T[["min","1%","50%","99%","max"]])

POSLE:
                      min        1%   50%        99%      max
Alcohol            0.0200  0.020000  4.13  13.929605  13.9345
Total expenditure  1.2522  1.254158  5.73  12.577131  12.5879

print("Ukupno NaN ćelija:", int(dataframe.isna().sum().sum()))

Ukupno NaN ćelija: 0

dataframe.head()

dataframe = pd.get_dummies(
    dataframe,
    columns=["Status"],
    drop_first=True
)

dataframe["Status_Developing"] = dataframe["Status_Developing"].astype(int)

dataframe["immunization_index"] = (
    dataframe["Hepatitis B"] +
    dataframe["Polio"] +
    dataframe["Diphtheria"]
) / 3

features_log = ["GDP","HIV/AIDS","infant deaths"]

for feature in features_log:
    dataframe[f"log_{feature}"] = np.log1p(dataframe[feature])

#Na svakom mestu gde vidite zelene komentare i ispod njih kod, taj kod je ili izmenjen ili dodat u celosti

dataframe["health_spending"] = dataframe["percentage expenditure"] * dataframe["log_GDP"]

dataframe["under_five_vaccination"] = dataframe["under-five deaths"] * (1 - dataframe["immunization_index"])

dataframe["malnutrition_index"] = dataframe["thinness 10-19 years"]  * (1 - dataframe["Income composition of resources"] )

dataframe.info()

<class 'pandas.core.frame.DataFrame'>
Index: 2912 entries, 15 to 2922
Data columns (total 28 columns):
 #   Column                           Non-Null Count  Dtype  
---  ------                           --------------  -----  
 0   Country                          2912 non-null   object 
 1   Year                             2912 non-null   int64  
 2   Life expectancy                  2912 non-null   float64
 3   infant deaths                    2912 non-null   int64  
 4   Alcohol                          2912 non-null   float64
 5   percentage expenditure           2912 non-null   float64
 6   Hepatitis B                      2912 non-null   float64
 7   Measles                          2912 non-null   int64  
 8   under-five deaths                2912 non-null   int64  
 9   Polio                            2912 non-null   float64
 10  Total expenditure                2912 non-null   float64
 11  Diphtheria                       2912 non-null   float64
 12  HIV/AIDS                         2912 non-null   float64
 13  GDP                              2912 non-null   float64
 14  Population                       2912 non-null   float64
 15  thinness 10-19 years             2912 non-null   float64
 16  Income composition of resources  2912 non-null   float64
 17  Schooling                        2912 non-null   float64
 18  Country_wb                       2912 non-null   object 
 19  Hepatitis B_was_missing          2912 non-null   int64  
 20  Status_Developing                2912 non-null   int64  
 21  immunization_index               2912 non-null   float64
 22  log_GDP                          2912 non-null   float64
 23  log_HIV/AIDS                     2912 non-null   float64
 24  log_infant deaths                2912 non-null   float64
 25  health_spending                  2912 non-null   float64
 26  under_five_vaccination           2912 non-null   float64
 27  malnutrition_index               2912 non-null   float64
dtypes: float64(20), int64(6), object(2)
memory usage: 659.8+ KB

X = dataframe.drop(["Life expectancy","Country"], axis = 1)
y = dataframe["Life expectancy"]

X = X.drop(columns=["Country_wb"])

X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.3,random_state=42,stratify=X["Status_Developing"])

X_val,X_test,y_val,y_test = train_test_split(X_temp,y_temp,test_size = 0.5,random_state = 42,stratify=X_temp["Status_Developing"])

plt.figure(figsize=(14, 10))
numeric_data = dataframe.select_dtypes(include=['float64', 'int64'])
correlation_matrix = numeric_data.corr()
sns.heatmap(correlation_matrix, annot=True, fmt='.2f', cmap='coolwarm')
plt.title('Correlation Matrix')
plt.show()

model = LinearRegression()

features =[
    "log_GDP",                 
    "Schooling",
    "log_infant deaths",       
    "Status_Developing",
    "Alcohol",
    "immunization_index",
    "log_HIV/AIDS",
    "health_spending",         
    "under_five_vaccination",  
    "malnutrition_index" 
]

target = "Life expectancy"

selected_features = []

print("FORWARD SELECTION REZULTATI\n")

def adjR2(r2,n,p):
    adj_r2 = 1 - (1 - r2) * (n - 1) / (n - p - 1)
    return adj_r2

for feature in features:
    
    selected_features.append(feature)
    
    model.fit(X_train[selected_features], y_train)
    
    y_train_pred = model.predict(X_train[selected_features])
    y_val_pred = model.predict(X_val[selected_features])
    
    train_rmse = np.sqrt(mean_squared_error(y_train, y_train_pred))
    val_rmse = np.sqrt(mean_squared_error(y_val, y_val_pred))
    
    train_r2 = r2_score(y_train, y_train_pred)
    val_r2 = r2_score(y_val, y_val_pred)

    n = X_val[selected_features].shape[0]
    p = X_val[selected_features].shape[1]

    adj_r2_val = adjR2(val_r2,n,p)
    
    print("Features:", selected_features)
    print("Train RMSE:", round(train_rmse, 3))
    print("Val RMSE:", round(val_rmse, 3))
    print("Train R2:", round(train_r2, 3))
    print("Val R2:", round(val_r2, 3))
    print("Adjusted Val R2:", round(adj_r2_val, 3))
    print("-" * 40)

FORWARD SELECTION REZULTATI

Features: ['log_GDP']
Train RMSE: 6.263
Val RMSE: 6.191
Train R2: 0.567
Val R2: 0.57
Adjusted Val R2: 0.569
----------------------------------------
Features: ['log_GDP', 'Schooling']
Train RMSE: 5.741
Val RMSE: 5.421
Train R2: 0.636
Val R2: 0.67
Adjusted Val R2: 0.669
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths']
Train RMSE: 5.635
Val RMSE: 5.338
Train R2: 0.649
Val R2: 0.68
Adjusted Val R2: 0.678
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing']
Train RMSE: 5.625
Val RMSE: 5.325
Train R2: 0.651
Val R2: 0.682
Adjusted Val R2: 0.679
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol']
Train RMSE: 5.522
Val RMSE: 5.101
Train R2: 0.663
Val R2: 0.708
Adjusted Val R2: 0.705
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol', 'immunization_index']
Train RMSE: 5.401
Val RMSE: 4.937
Train R2: 0.678
Val R2: 0.726
Adjusted Val R2: 0.723
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol', 'immunization_index', 'log_HIV/AIDS']
Train RMSE: 3.736
Val RMSE: 3.436
Train R2: 0.846
Val R2: 0.867
Adjusted Val R2: 0.865
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol', 'immunization_index', 'log_HIV/AIDS', 'health_spending']
Train RMSE: 3.735
Val RMSE: 3.432
Train R2: 0.846
Val R2: 0.868
Adjusted Val R2: 0.865
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol', 'immunization_index', 'log_HIV/AIDS', 'health_spending', 'under_five_vaccination']
Train RMSE: 3.735
Val RMSE: 3.43
Train R2: 0.846
Val R2: 0.868
Adjusted Val R2: 0.865
----------------------------------------
Features: ['log_GDP', 'Schooling', 'log_infant deaths', 'Status_Developing', 'Alcohol', 'immunization_index', 'log_HIV/AIDS', 'health_spending', 'under_five_vaccination', 'malnutrition_index']
Train RMSE: 3.721
Val RMSE: 3.414
Train R2: 0.847
Val R2: 0.869
Adjusted Val R2: 0.866
----------------------------------------

#Dodat malnutrition_index kao feature
final_features =[
    'log_GDP',
    'Schooling',
    'Alcohol',
    'immunization_index',
    'log_HIV/AIDS',
    'malnutrition_index',
]

model.fit(X_train[final_features], y_train)

y_train_pred = model.predict(X_train[final_features])
y_val_pred = model.predict(X_val[final_features])

train_rmse = np.sqrt(mean_squared_error(y_train, y_train_pred))
val_rmse = np.sqrt(mean_squared_error(y_val, y_val_pred))

train_r2 = r2_score(y_train, y_train_pred)
val_r2 = r2_score(y_val, y_val_pred)

n = X_val[final_features].shape[0]
p = X_val[final_features].shape[1]

adj_r2_val = adjR2(val_r2,n,p)

print("Features:", final_features)
print("Train RMSE:", round(train_rmse, 3))
print("Val RMSE:", round(val_rmse, 3))
print("Train R2:", round(train_r2, 3))
print("Val R2:", round(val_r2, 3))
print("Adjusted Val R2:", round(adj_r2_val, 3))
print("-" * 40)

Features: ['log_GDP', 'Schooling', 'Alcohol', 'immunization_index', 'log_HIV/AIDS', 'malnutrition_index']
Train RMSE: 3.788
Val RMSE: 3.509
Train R2: 0.842
Val R2: 0.862
Adjusted Val R2: 0.86
----------------------------------------

X = dataframe[final_features]
X = sm.add_constant(X)

vif_data = pd.DataFrame()
vif_data["Feature"] = X.columns
vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]

print(vif_data)

              Feature        VIF
0               const  77.514018
1             log_GDP   2.870062
2           Schooling   3.136786
3             Alcohol   1.669635
4  immunization_index   1.279039
5        log_HIV/AIDS   1.410169
6  malnutrition_index   1.790417

y_test_pred = model.predict(X_test[final_features])

test_rmse = np.sqrt(mean_squared_error(y_test,y_test_pred))
test_mae = mean_absolute_error(y_test,y_test_pred)
test_r2 = r2_score(y_test,y_test_pred)

print("TEST RMSE :",test_rmse)
print("TEST MAE : ",test_mae)
print("TEST R2",test_r2)

TEST RMSE : 3.343433367723299
TEST MAE :  2.6586184075905805
TEST R2 0.8709971313736222

ss = StandardScaler()

X_train_scaled = ss.fit_transform(X_train)
X_test_scaled = ss.transform(X_test)

alphas = [0.001, 0.01, 0.1, 1, 10, 100]

ridge_cv = RidgeCV(alphas=alphas)
ridge_cv.fit(X_train_scaled, y_train)

best_ridge_alpha = ridge_cv.alpha_

print("Best alpha Ridge:", best_ridge_alpha)

lasso_cv = LassoCV(alphas=alphas, max_iter=10000)
lasso_cv.fit(X_train_scaled, y_train)

best_lasso_alpha = lasso_cv.alpha_

print("Best alpha Lasso:", best_lasso_alpha)

Best alpha Ridge: 0.1
Best alpha Lasso: 0.001

ridge = Ridge(alpha=best_ridge_alpha, max_iter=10000)   
ridge.fit(X_train_scaled, y_train)

y_pred_ridge = ridge.predict(X_test_scaled)

print("Ridge RMSE:",np.sqrt(mean_squared_error(y_test, y_pred_ridge)))
print("Ridge MAE:", mean_absolute_error(y_test, y_pred_ridge))

Ridge RMSE: 3.111915197701377
Ridge MAE: 2.349573852780147

lasso = Lasso(alpha=best_lasso_alpha, max_iter=10000)
lasso.fit(X_train_scaled, y_train)

y_pred_lasso = lasso.predict(X_test_scaled)

print("Lasso RMSE:",np.sqrt(mean_squared_error(y_test, y_pred_lasso)))
print("Lasso MAE:", mean_absolute_error(y_test, y_pred_lasso))

Lasso RMSE: 3.1082145637371723
Lasso MAE: 2.349374310587357

rf = RandomForestRegressor(n_estimators=200,max_depth=None,random_state=42)

rf.fit(X_train, y_train)

y_pred_rf = rf.predict(X_test)

print("Random forest RMSE:",np.sqrt(mean_squared_error(y_test, y_pred_rf)))
print("Random forest MAE:", mean_absolute_error(y_test, y_pred_rf))

Random forest RMSE: 1.6550497705915062
Random forest MAE: 1.1549393592677293

### DODAT KOD ###
feature_names = X_train.columns

rf_percent = 100 * pd.Series(
    rf.feature_importances_,
    index=feature_names
).sort_values(ascending=False)

max_val = rf_percent.head(10).max()

rf_percent.head(10).plot(kind='barh')
plt.title("Random Forest Feature Importance (%)")
plt.xlabel("Procenat doprinosa")
plt.xticks(np.arange(0, max_val + 5, 5))
plt.gca().invert_yaxis()
plt.show()

xgb = XGBRegressor(n_estimators=200,learning_rate=0.05,max_depth=4,subsample=0.8,colsample_bytree=0.8,random_state=42)

xgb.fit(X_train, y_train)

y_pred_xgb = xgb.predict(X_test)

print("XGBoost RMSE:",np.sqrt(mean_squared_error(y_test, y_pred_xgb)))
print("XGBoost MAE:", mean_absolute_error(y_test, y_pred_xgb))

XGBoost RMSE: 1.829879908285139
XGBoost MAE: 1.3588102065725784

### DODAT KOD ###
xgboost_percent = 100 * pd.Series(
    xgb.feature_importances_,
    index=feature_names
).sort_values(ascending=False)

max_val = xgboost_percent.head(10).max()

xgboost_percent.head(10).plot(kind='barh')
plt.title("Random Forest Feature Importance (%)")
plt.xlabel("Procenat doprinosa")
plt.xticks(np.arange(0, max_val + 5, 5))
plt.gca().invert_yaxis()
plt.show()

	Country	Alcohol
874	Estonia	17.87
228	Belarus	17.31
873	Estonia	16.99
875	Estonia	16.58
227	Belarus	16.35
876	Estonia	15.52
1523	Lithuania	15.19
1525	Lithuania	15.14
877	Estonia	15.07
872	Estonia	15.04
1524	Lithuania	15.04

	Country	Measles
1908	Nigeria	212183
731	Democratic Republic of the Congo	182485
1907	Nigeria	168107
1905	Nigeria	141258
725	Democratic Republic of the Congo	133802
567	China	131441
570	China	124219
1575	Malawi	118712
1903	Nigeria	110927
568	China	109023

	Country	GDP
1539	Luxembourg	119172.74180
1542	Luxembourg	115761.57700
1545	Luxembourg	114293.84330
1540	Luxembourg	113751.85000
1547	Luxembourg	89739.71170
2074	Qatar	88564.82298
2525	Switzerland	87998.44468
1915	Norway	87646.75346
2072	Qatar	86852.71190
2075	Qatar	85948.74600
2522	Switzerland	85814.58857
1918	Norway	85128.65759
2523	Switzerland	84658.88768
2524	Switzerland	83164.38795
2078	Qatar	82967.37228
1549	Luxembourg	75716.35180
2526	Switzerland	74276.71842
1919	Norway	74114.69715
2528	Switzerland	72119.56870
2527	Switzerland	69672.47100
1178	Iceland	68348.31817
114	Australia	67792.33860
115	Australia	67677.63477
1920	Norway	66775.39440
2071	Qatar	66346.52267
1550	Luxembourg	65445.88530
744	Denmark	64322.66640
2529	Switzerland	63223.46778
738	Denmark	62425.53920
116	Australia	62245.12900
113	Australia	62214.69120
741	Denmark	61753.66700
2077	Qatar	61478.23813
1258	Ireland	61388.17457
1257	Ireland	61235.41500
739	Denmark	61191.19263

	Country	Year	GDP	GDP_diff
1539	Luxembourg	2014	119172.74180	117972.91950
1546	Luxembourg	2007	1618.49280	112675.35050
1545	Luxembourg	2008	114293.84330	101095.17400
1543	Luxembourg	2010	14965.36100	100796.21600
1542	Luxembourg	2011	115761.57700	99012.44100
1541	Luxembourg	2012	16749.13600	97002.71400
1547	Luxembourg	2006	89739.71170	88121.21890
1916	Norway	2009	817.77681	86828.97665
1915	Norway	2010	87646.75346	86071.76736
2076	Qatar	2010	736.22784	85212.51816
2079	Qatar	2007	675.61258	82291.75970
1548	Luxembourg	2005	8289.69641	81450.01529
2074	Qatar	2012	88564.82298	79729.94340
2073	Qatar	2013	8834.87958	78017.83232
2522	Switzerland	2014	85814.58857	76824.74617
1918	Norway	2007	85128.65759	75440.06149
1549	Luxembourg	2004	75716.35180	67426.65539
1179	Iceland	2006	5613.54115	62734.77702
1921	Norway	2004	5757.26916	61018.12524
2077	Qatar	2009	61478.23813	60742.01029

	feature	pvalue	mean_GDP_missing	mean_GDP_present	n_missing	n_present
5	percentage expenditure	2.371185e-214	0.000000e+00	8.721280e+02	433	2487
18	Schooling	1.356547e-05	1.122500e+01	1.208170e+01	288	2487
17	Income composition of resources	4.227555e-05	5.952160e-01	6.312870e-01	287	2484
2	Adult Mortality	9.153265e-05	1.758326e+02	1.619049e+02	430	2482
14	Population	1.124979e-04	3.455477e+07	3.734710e+07	433	2487
3	infant deaths	1.615108e-04	2.429330e+01	3.124889e+01	433	2487
9	under-five deaths	5.857122e-04	3.292610e+01	4.340772e+01	433	2487
13	HIV/AIDS	1.449060e-03	9.505774e-01	1.887053e+00	433	2487
10	Polio	3.304915e-02	8.439059e+01	8.256220e+01	425	2476
6	Hepatitis B	4.574983e-02	8.201355e+01	8.078419e+01	369	2011
1	Life expectancy	7.325170e-02	6.887186e+01	6.938864e+01	430	2482
4	Alcohol	1.408776e-01	4.923526e+00	4.581734e+00	397	2330
11	Total expenditure	2.380133e-01	6.305794e+00	5.879074e+00	378	2332
12	Diphtheria	2.802968e-01	8.296941e+01	8.254281e+01	425	2476
15	thinness 10-19 years	3.216244e-01	4.825296e+00	4.830248e+00	423	2463
7	Measles	5.028844e-01	2.712603e+03	2.350744e+03	433	2487
16	thinness 5-9 years	6.450885e-01	4.792199e+00	4.873244e+00	423	2463
0	Year	7.003569e-01	2.007594e+03	2.007501e+03	433	2487
8	BMI	9.749567e-01	3.814846e+01	3.844458e+01	423	2463

	Country	Year	Status	Life expectancy	Adult Mortality	infant deaths	Alcohol	percentage expenditure	Hepatitis B	Measles	...	Polio	Total expenditure	Diphtheria	HIV/AIDS	GDP	Population	thinness 10-19 years	thinness 5-9 years	Income composition of resources	Schooling
0	Afghanistan	2015	Developing	65.0	263.0	62	0.01	71.279624	65.0	1154	...	6.0	8.16	65.0	0.1	584.259210	33736494.0	17.2	17.3	0.479	10.1
1	Afghanistan	2014	Developing	59.9	271.0	64	0.01	73.523582	62.0	492	...	58.0	8.18	62.0	0.1	612.696514	327582.0	17.5	17.5	0.476	10.0
2	Afghanistan	2013	Developing	59.9	268.0	66	0.01	73.219243	64.0	430	...	62.0	8.13	64.0	0.1	631.744976	31731688.0	17.7	17.7	0.470	9.9
3	Afghanistan	2012	Developing	59.5	272.0	69	0.01	78.184215	67.0	2787	...	67.0	8.52	67.0	0.1	669.959000	3696958.0	17.9	18.0	0.463	9.8
4	Afghanistan	2011	Developing	59.2	275.0	71	0.01	7.097109	68.0	3013	...	68.0	7.87	68.0	0.1	63.537231	2978599.0	18.2	18.2	0.454	9.5

	count	unique	top	freq	mean	std	min	25%	50%	75%	max
Country	2938	193	Afghanistan	16	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Year	2938.0	NaN	NaN	NaN	2007.51872	4.613841	2000.0	2004.0	2008.0	2012.0	2015.0
Status	2938	2	Developing	2426	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Life expectancy	2928.0	NaN	NaN	NaN	69.224932	9.523867	36.3	63.1	72.1	75.7	89.0
Adult Mortality	2928.0	NaN	NaN	NaN	164.796448	124.292079	1.0	74.0	144.0	228.0	723.0
infant deaths	2938.0	NaN	NaN	NaN	30.303948	117.926501	0.0	0.0	3.0	22.0	1800.0
Alcohol	2744.0	NaN	NaN	NaN	4.602861	4.052413	0.01	0.8775	3.755	7.7025	17.87
percentage expenditure	2938.0	NaN	NaN	NaN	738.251295	1987.914858	0.0	4.685343	64.912906	441.534144	19479.91161
Hepatitis B	2385.0	NaN	NaN	NaN	80.940461	25.070016	1.0	77.0	92.0	97.0	99.0
Measles	2938.0	NaN	NaN	NaN	2419.59224	11467.272489	0.0	0.0	17.0	360.25	212183.0
BMI	2904.0	NaN	NaN	NaN	38.321247	20.044034	1.0	19.3	43.5	56.2	87.3
under-five deaths	2938.0	NaN	NaN	NaN	42.035739	160.445548	0.0	0.0	4.0	28.0	2500.0
Polio	2919.0	NaN	NaN	NaN	82.550188	23.428046	3.0	78.0	93.0	97.0	99.0
Total expenditure	2712.0	NaN	NaN	NaN	5.93819	2.49832	0.37	4.26	5.755	7.4925	17.6
Diphtheria	2919.0	NaN	NaN	NaN	82.324084	23.716912	2.0	78.0	93.0	97.0	99.0
HIV/AIDS	2938.0	NaN	NaN	NaN	1.742103	5.077785	0.1	0.1	0.1	0.8	50.6
GDP	2490.0	NaN	NaN	NaN	7483.158469	14270.169342	1.68135	463.935626	1766.947595	5910.806335	119172.7418
Population	2286.0	NaN	NaN	NaN	12753375.120052	61012096.508428	34.0	195793.25	1386542.0	7420359.0	1293859294.0
thinness 10-19 years	2904.0	NaN	NaN	NaN	4.839704	4.420195	0.1	1.6	3.3	7.2	27.7
thinness 5-9 years	2904.0	NaN	NaN	NaN	4.870317	4.508882	0.1	1.5	3.3	7.2	28.6
Income composition of resources	2771.0	NaN	NaN	NaN	0.627551	0.210904	0.0	0.493	0.677	0.779	0.948
Schooling	2775.0	NaN	NaN	NaN	11.992793	3.35892	0.0	10.1	12.3	14.3	20.7

	Country	Population
1187	India	1.293859e+09
1194	India	1.179681e+09
1195	India	1.161978e+09
1196	India	1.144119e+09
1197	India	1.126136e+09

	Country	Population
560	China	137122.0
561	China	136427.0
562	China	135738.0
563	China	135695.0
564	China	134413.0
565	China	133775.0
566	China	133126.0
567	China	1324655.0
568	China	1317885.0
569	China	13112.0
570	China	13372.0
571	China	129675.0
572	China	12884.0
573	China	1284.0
574	China	127185.0
575	China	1262645.0

	Country	Income composition of resources
74	Antigua and Barbuda	0.0
75	Antigua and Barbuda	0.0
76	Antigua and Barbuda	0.0
77	Antigua and Barbuda	0.0
78	Antigua and Barbuda	0.0
...	...	...
2853	Vanuatu	0.0
2854	Vanuatu	0.0
2855	Vanuatu	0.0
2856	Vanuatu	0.0
2857	Vanuatu	0.0

	Country	Year	Status	Life expectancy	infant deaths	Alcohol	percentage expenditure	Hepatitis B	Measles	under-five deaths	...	Total expenditure	Diphtheria	HIV/AIDS	GDP	Population	thinness 10-19 years	Income composition of resources	Schooling	Country_wb
15	Afghanistan	2000	Developing	54.8	88	0.02	10.424960	62.0	6532	122	...	8.20	24.0	0.1	114.560000	20130327.0	2.3	0.338	5.5	Afghanistan
14	Afghanistan	2001	Developing	55.3	88	0.02	10.574728	63.0	8762	122	...	7.80	33.0	0.1	117.496980	20284307.0	2.1	0.340	5.9	Afghanistan
13	Afghanistan	2002	Developing	56.2	88	0.02	16.887351	64.0	2486	122	...	7.76	36.0	0.1	187.845950	21378117.0	19.9	0.341	6.2	Afghanistan
12	Afghanistan	2003	Developing	56.7	87	0.02	11.089053	65.0	798	122	...	8.82	41.0	0.1	198.728544	22733049.0	19.7	0.373	6.5	Afghanistan
11	Afghanistan	2004	Developing	57.0	87	0.02	15.296066	67.0	466	120	...	8.79	5.0	0.1	219.141353	23560654.0	19.5	0.381	6.8	Afghanistan

Model	RMSE	MAE
Linear Regression	3.26	2.59
Ridge	3.02	2.29
Lasso	3.02	2.29
Random Forest	1.62	1.11
XGBoost	1.83	1.33

Life Expectancy (WHO)

Uvod i opis problema¶

Opis problema¶

Početna konfiguracija¶

Učitavanje podataka¶

Exploratory Data Analaysis¶

Univarijatna analiza¶

Life Expectancy¶

GDP¶

HIV/AIDS¶

BMI¶

percentage expenditure¶

thinness 10-19 years¶

Bivarijantna analiza¶

COUNTRY¶

YEAR¶

Status¶

Adult Mortality¶

infant deaths¶

ALCOHOL¶

PERCENTAGE EXPENDITURE¶

Hepatitis B¶

MEASLES¶

BMI¶

under-five deaths¶

POLIO¶

TOTAL EXPENDITURE¶

DIPTHTHERIA¶

HIV/AIDS¶

GDP¶

POPULATION¶

thinness 1-19 years¶

THINNESS 5-9 YEARS¶

INCOME COMPOSITION OF RESOURCES¶

SCHOOLING¶

Statisticki testovi¶

Statistički testovi u bivarijantnoj analizi¶

Spearman-ova korelacija

Rezultati Spearman-ove korelacije¶

Mann–Whitney U test: Status i očekivani životni vek

Povezanost između godine i očekivanog životnog veka

Napomena o tumačenju rezultata

ČIŠĆENJE PODATAKA¶

Korelacija nedostajućih vrednosti¶

POPULATION¶

Osnovna statistika – Population¶

GDP¶

HEPATITIS B¶

OSTALE PROMENLJIVE¶

IZUZECI¶

Feature Engineering¶

Data preprocessing¶

Feature Selection i procena modela¶

Matrica korelacije¶

Forward-selection¶

Implementacija ostalih modela i poredjenje¶

Ridge i Lasso Regularizacija¶

Poredjenje modela¶

Zaključak¶

Reference¶